This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.

Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Ctrl+Shift+Enter.

suppressMessages(library(tidyverse))
suppressMessages(library(stringr))
suppressMessages(library(ISLR))
suppressMessages(library(caret))
suppressMessages(library(doMC))
library(plotly)
registerDoMC(cores=4)

Add a new chunk by clicking the Insert Chunk button on the toolbar or by pressing Ctrl+Alt+I.

When you save the notebook, an HTML file containing the code and output will be saved alongside it (click the Preview button or press Ctrl+Shift+K to preview the HTML file).

Getting and proccesing the data

library(stringr)
myData = read.csv('/home/harpo/Dropbox/ongoing-work/git-repos/labeling-datasets/phd/datasets/data_all_result.txt', stringsAsFactors = F, sep = ' ')

#Create data backup
myData.bkup <- myData
#Create new column: length of model, and number of periodicity, duration and size characteristic in the model.
myData = myData %>% mutate(letter_count = nchar(State))
#Periodicity
myData = myData %>% mutate(strong_p = str_count(State,'[a-i]'))
myData = myData %>% mutate(weak_p = str_count(State,'[A-I]'))
myData = myData %>% mutate(weak_np = str_count(State,'[r-z]'))
myData = myData %>% mutate(strong_np = str_count(State,'[R-Z]'))
#Duration
myData = myData %>% mutate(duration_s = str_count(State,'(a|A|r|R|1|d|D|u|U|4|g|G|x|X|7)'))#c('a','A','r','R','1','d','D','u','U','4','g','G','x','X','7')))
myData = myData %>% mutate(duration_m = str_count(State,'(b|B|s|S|2|e|E|v|V|5|h|H|y|Y|8)'))#c('b','B','s','S','2','e','E','v','V','5','h','H','y','Y','8')))
myData = myData %>% mutate(duration_l = str_count(State,'(c|C|t|T|3|f|F|w|W|6|i|I|z|Z|9)'))#c('c','C','t','T','3','f','F','w','W','6','i','I','z','Z','9')))
#Size
myData = myData %>% mutate(size_s = str_count(State,'[a-c]') + str_count(State,'[A-C]') + str_count(State,'[r-t]') + str_count(State,'[R-T]') + str_count(State,'[1-3]'))
myData = myData %>% mutate(size_m = str_count(State,'[d-f]') + str_count(State,'[D-F]') + str_count(State,'[u-w]') + str_count(State,'[U-W]') + str_count(State,'[4-6]'))
myData = myData %>% mutate(size_l = str_count(State,'[g-i]') + str_count(State,'[G-I]') + str_count(State,'[x-z]') + str_count(State,'[X-Z]') + str_count(State,'[7-9]'))

#Remove from LabelName unnecessary characters (ej: V42, -17)
myData <- myData %>% mutate(LabelName = gsub('V[0-9]+-','',LabelName))
myData <- myData %>% mutate(LabelName = gsub('-[0-9]+','',LabelName))
myData <- myData %>% mutate(LabelName = gsub('CC[0-9]+-','CC-',LabelName))

#Keep only connection with more than 3 symbols
myData <- myData %>% filter(letter_count > 3)

#Periodicity %
myData <- myData %>% mutate(strong_p = (strong_p / letter_count))
myData <- myData %>% mutate(weak_p = (weak_p / letter_count))
myData <- myData %>% mutate(strong_np = (strong_np / letter_count))
myData <- myData %>% mutate(weak_np = (weak_np / letter_count))
#Duration %
myData <- myData %>% mutate(duration_s = (duration_s / letter_count))
myData <- myData %>% mutate(duration_m = (duration_m / letter_count))
myData <- myData %>% mutate(duration_l = (duration_l / letter_count))
#Size %
myData <- myData %>% mutate(size_s = (size_s / letter_count))
myData <- myData %>% mutate(size_m = (size_m / letter_count))
myData <- myData %>% mutate(size_l = (size_l / letter_count))

#head(myData)
myData[1:20,]

#Making feature vectors
feature_vectors = myData[,c('strong_p','weak_p','weak_np','strong_np','duration_s','duration_m','duration_l','size_s','size_m','size_l','letter_count','Label','LabelName')]
names(feature_vectors) = c("sp","wp","wnp","snp","ds","dm","dl","ss","sm","sl","length","class","subclass")
feature_vectors$class = factor(feature_vectors$class)
feature_vectors$subclass = factor(feature_vectors$subclass)

Create training set and testset

set.seed(300)
trainIndex <- createDataPartition(feature_vectors$class, p=0.80, list=FALSE)
data_train <- feature_vectors[ trainIndex,]
data_test <- feature_vectors[-trainIndex,]
#data_train = data_train %>% filter(length>5)
train <- upSample(x = data_train,  y = data_train$class, yname="class")
#train <- upSample(x = train,  y = train$subclass, yname="class")
training <- train[,-c(11,12)]
testing <- data_test[,-c(11)]
training
ctrl_fast <- trainControl(method="cv", 
                     repeats=2,
                     number=10, 
                     summaryFunction=twoClassSummary,
                     verboseIter=T,
                     classProbs=TRUE,
                     allowParallel = TRUE)  
ctrl <- trainControl(method="repeatedcv",repeats = 3) #,classProbs=TRUE,summaryFunction = twoClassSummary)

Random Forest Classificator

  # Random Forest
rfFit <- train(class ~ .,
               data = training,
               metric="ROC",
               method = "rf",
               trControl = ctrl_fast)

rfFit
rfFit$finalModel
predsrfprobs=predict(rfFit,testing,type='prob')
predsrf=ifelse(predsrfprobs$Botnet >=0.9,'Botnet','Normal')
confusionMatrix(predsrf,testing$class)
library(ggplot2)
library(plotROC)
selectedIndices <- rfFit$pred$mtry == 2
ggplot(cbind(predsrfprobs,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

cbind(predsrfprobs,class=testing$class)

KNN

#Checking distibution in origanl data and partitioned data
prop.table(table(training$class)) * 100

Botnet Normal 
    50     50 
prop.table(table(testing$class)) * 100

  Botnet   Normal 
69.25986 30.74014 
prop.table(table(feature_vectors$class)) * 100

 Botnet  Normal 
69.2449 30.7551 
trainX <- training[,names(training) != "class"]
preProcValues <- preProcess(x = trainX,method = c("center", "scale"))
preProcValues
Created from 10256 samples and 11 variables

Pre-processing:
  - centered (10)
  - ignored (1)
  - scaled (10)
knnFit <- train(class ~ ., data = training, method = "knn", trControl = ctrl_fast, preProcess = c("center","scale"), tuneLength = 20)
The metric "Accuracy" was not in the result set. ROC will be used instead.
+ Fold01: k= 5 
- Fold01: k= 5 
+ Fold01: k= 7 
- Fold01: k= 7 
+ Fold01: k= 9 
- Fold01: k= 9 
+ Fold01: k=11 
- Fold01: k=11 
+ Fold01: k=13 
- Fold01: k=13 
+ Fold01: k=15 
- Fold01: k=15 
+ Fold01: k=17 
- Fold01: k=17 
+ Fold01: k=19 
- Fold01: k=19 
+ Fold01: k=21 
- Fold01: k=21 
+ Fold01: k=23 
- Fold01: k=23 
+ Fold01: k=25 
- Fold01: k=25 
+ Fold01: k=27 
- Fold01: k=27 
+ Fold01: k=29 
- Fold01: k=29 
+ Fold01: k=31 
- Fold01: k=31 
+ Fold01: k=33 
- Fold01: k=33 
+ Fold01: k=35 
- Fold01: k=35 
+ Fold01: k=37 
- Fold01: k=37 
+ Fold01: k=39 
- Fold01: k=39 
+ Fold01: k=41 
- Fold01: k=41 
+ Fold01: k=43 
- Fold01: k=43 
+ Fold02: k= 5 
- Fold02: k= 5 
+ Fold02: k= 7 
- Fold02: k= 7 
+ Fold02: k= 9 
- Fold02: k= 9 
+ Fold02: k=11 
- Fold02: k=11 
+ Fold02: k=13 
- Fold02: k=13 
+ Fold02: k=15 
- Fold02: k=15 
+ Fold02: k=17 
- Fold02: k=17 
+ Fold02: k=19 
- Fold02: k=19 
+ Fold02: k=21 
- Fold02: k=21 
+ Fold02: k=23 
- Fold02: k=23 
+ Fold02: k=25 
- Fold02: k=25 
+ Fold02: k=27 
- Fold02: k=27 
+ Fold02: k=29 
- Fold02: k=29 
+ Fold02: k=31 
- Fold02: k=31 
+ Fold02: k=33 
- Fold02: k=33 
+ Fold02: k=35 
- Fold02: k=35 
+ Fold02: k=37 
- Fold02: k=37 
+ Fold02: k=39 
- Fold02: k=39 
+ Fold02: k=41 
- Fold02: k=41 
+ Fold02: k=43 
- Fold02: k=43 
+ Fold03: k= 5 
- Fold03: k= 5 
+ Fold03: k= 7 
- Fold03: k= 7 
+ Fold03: k= 9 
- Fold03: k= 9 
+ Fold03: k=11 
- Fold03: k=11 
+ Fold03: k=13 
- Fold03: k=13 
+ Fold03: k=15 
- Fold03: k=15 
+ Fold03: k=17 
- Fold03: k=17 
+ Fold03: k=19 
- Fold03: k=19 
+ Fold03: k=21 
- Fold03: k=21 
+ Fold03: k=23 
- Fold03: k=23 
+ Fold03: k=25 
- Fold03: k=25 
+ Fold03: k=27 
- Fold03: k=27 
+ Fold03: k=29 
- Fold03: k=29 
+ Fold03: k=31 
- Fold03: k=31 
+ Fold03: k=33 
- Fold03: k=33 
+ Fold03: k=35 
- Fold03: k=35 
+ Fold03: k=37 
- Fold03: k=37 
+ Fold03: k=39 
- Fold03: k=39 
+ Fold03: k=41 
- Fold03: k=41 
+ Fold03: k=43 
- Fold03: k=43 
+ Fold04: k= 5 
- Fold04: k= 5 
+ Fold04: k= 7 
- Fold04: k= 7 
+ Fold04: k= 9 
- Fold04: k= 9 
+ Fold04: k=11 
- Fold04: k=11 
+ Fold04: k=13 
- Fold04: k=13 
+ Fold04: k=15 
- Fold04: k=15 
+ Fold04: k=17 
- Fold04: k=17 
+ Fold04: k=19 
- Fold04: k=19 
+ Fold04: k=21 
- Fold04: k=21 
+ Fold04: k=23 
- Fold04: k=23 
+ Fold04: k=25 
- Fold04: k=25 
+ Fold04: k=27 
- Fold04: k=27 
+ Fold04: k=29 
- Fold04: k=29 
+ Fold04: k=31 
- Fold04: k=31 
+ Fold04: k=33 
- Fold04: k=33 
+ Fold04: k=35 
- Fold04: k=35 
+ Fold04: k=37 
- Fold04: k=37 
+ Fold04: k=39 
- Fold04: k=39 
+ Fold04: k=41 
- Fold04: k=41 
+ Fold04: k=43 
- Fold04: k=43 
+ Fold05: k= 5 
- Fold05: k= 5 
+ Fold05: k= 7 
- Fold05: k= 7 
+ Fold05: k= 9 
- Fold05: k= 9 
+ Fold05: k=11 
- Fold05: k=11 
+ Fold05: k=13 
- Fold05: k=13 
+ Fold05: k=15 
- Fold05: k=15 
+ Fold05: k=17 
- Fold05: k=17 
+ Fold05: k=19 
- Fold05: k=19 
+ Fold05: k=21 
- Fold05: k=21 
+ Fold05: k=23 
- Fold05: k=23 
+ Fold05: k=25 
- Fold05: k=25 
+ Fold05: k=27 
- Fold05: k=27 
+ Fold05: k=29 
- Fold05: k=29 
+ Fold05: k=31 
- Fold05: k=31 
+ Fold05: k=33 
- Fold05: k=33 
+ Fold05: k=35 
- Fold05: k=35 
+ Fold05: k=37 
- Fold05: k=37 
+ Fold05: k=39 
- Fold05: k=39 
+ Fold05: k=41 
- Fold05: k=41 
+ Fold05: k=43 
- Fold05: k=43 
+ Fold06: k= 5 
These variables have zero variances: subclassFrom-Botnet-TCP-CC-HTTP-Custom-Port-Not-Encrypted-Non-Periodic
- Fold06: k= 5 
+ Fold06: k= 7 
These variables have zero variances: subclassFrom-Botnet-TCP-CC-HTTP-Custom-Port-Not-Encrypted-Non-Periodic
- Fold06: k= 7 
+ Fold06: k= 9 
These variables have zero variances: subclassFrom-Botnet-TCP-CC-HTTP-Custom-Port-Not-Encrypted-Non-Periodic
- Fold06: k= 9 
+ Fold06: k=11 
These variables have zero variances: subclassFrom-Botnet-TCP-CC-HTTP-Custom-Port-Not-Encrypted-Non-Periodic
- Fold06: k=11 
+ Fold06: k=13 
These variables have zero variances: subclassFrom-Botnet-TCP-CC-HTTP-Custom-Port-Not-Encrypted-Non-Periodic
- Fold06: k=13 
+ Fold06: k=15 
These variables have zero variances: subclassFrom-Botnet-TCP-CC-HTTP-Custom-Port-Not-Encrypted-Non-Periodic
#Plotting yields Number of Neighbours Vs accuracy (based on repeated cross validation)
plot(knnFit)
knnPredict <- predict(knnFit,newdata = testing )
#Get the confusion matrix to see accuracy value and other parameter values
confusionMatrix(knnPredict, testing$class )
mean(knnPredict == testing$class)
library(pROC)
knnPredict <- predict(knnFit,newdata = testing , type="prob")
knnROC <- roc(testing$class,knnPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))
knnROC
ggplot(cbind(knnPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

#plot(knnROC, type="S", print.thres= 0.5)

Logistic Regression

logicRFit <- train(class ~ ., method='glm', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))
#logicRFit <- train(class ~ sp*wp*wnp*snp*ds*dm*dl*ss*sm*sl, method='glm', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))
#logicRFit <- train(class ~ sp+wp+wnp+snp+ds+dm+dl+ss+sm+sl, method='glm', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))

#summary(logicRFit)
#Output of Logistic Regression fit
logicRFit

logicRPredict <- predict(logicRFit, newdata = testing )

confusionMatrix(logicRPredict, testing$class)
logicRPredict <- predict(logicRFit, newdata = testing, type="prob")
logicROC <- roc(testing$class,logicRPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))

ggplot(cbind(logicRPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

#logicROC

Naive Bayes

naiveBayesFit <- train(class ~ ., method='nb', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training)
naiveBayesFit
naiveBayesPredict <- predict(naiveBayesFit, newdata = testing)

confusionMatrix(naiveBayesPredict, testing$class)
naiveBayesPredict <- predict(naiveBayesFit, newdata = testing, type = 'prob')
naiveBayesROC <- roc(testing$class,naiveBayesPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))
naiveBayesROC

ggplot(cbind(naiveBayesPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

#plot(naiveBayesROC, type="S", print.thres= 0.5)

Suport Vector Machine

svmFit <- train(class ~ ., method='svmLinear', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))
svmFit
svmPredict <- predict(svmFit, newdata = testing)
confusionMatrix(svmPredict, testing$class)
svmPredict <- predict(svmFit, newdata = testing, type = "prob")

svmROC <- roc(testing$class,svmPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))
svmROC

ggplot(cbind(svmPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

Comparing Models

resamps <- resamples(list(rf = rfFit, lr = logicRFit, nv = naiveBayesFit, svm = svmFit))
Error in resamples(list(rf = rfFit, lr = logicRFit, nv = naiveBayesFit,  : 
  objeto 'rfFit' no encontrado

Making probabilistic table.

#load("./botnet_prob_results.Rda")
library(grid)
library(gridExtra)
#botnet_prob_result %>% group_by(subclass) %>% summarise(n=n(),mean=mean(KNN),sd=sd(KNN)) %>% arrange(desc(n))
#botnet_prob_result %>% group_by(subclass) %>% summarise(n=n(),mean=mean(NaiveBayes),sd=sd(NaiveBayes)) %>% arrange(desc(n))%>% top_n(10)
botnet_10_top<-botnet_prob_result %>% filter(TrueClass=="Normal") %>% group_by(subclass) %>% summarise(n=n()) %>% arrange(desc(n))%>% top_n(20)
Selecting by n
botnet_10_top<-inner_join(botnet_10_top,botnet_prob_result,by="subclass")
knn_plot<-bwplot(subclass~KNN,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
rf_plot<-bwplot(subclass~RamdomForest,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
rl_plot<-bwplot(subclass~LogisticRegression,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
svm_plot<-bwplot(subclass~SVM,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
pl = list(knn_plot, rf_plot,rl_plot,svm_plot)
# do.call(grid.arrange, c(pl, nrow=1))
do.call(grid.arrange, c(lapply(pl, update), list(nrow=1)))

some heatmaps

library(scales)
knn_m<-matrix(botnet_prob_result[1:1830,]$KNN,ncol=30,nrow=61)
svm_m<-matrix(botnet_prob_result[1:1830,]$SVM,ncol=30,nrow=61)
lr_m<-matrix(botnet_prob_result[1:1830,]$LogisticRegression,ncol=30,nrow=61)
nb_m<-matrix(botnet_prob_result[1:1830,]$NaiveBayes,ncol=30,nrow=61)
rf_m<-matrix(botnet_prob_result[1:1830,]$RamdomForest,ncol=30,nrow=61)
mdf<-as.data.frame(knn_m)
mdf<-cbind(mdf,id=seq(1:61))
mdf<-reshape2::melt(mdf,id.vars=c("id"))
h1<-ggplot(mdf)+
  geom_tile(aes(x=id,y=variable,fill=value),
            colour = "white") +
  scale_fill_gradient(low = "white",
    high = "orange")+ylab("")+xlab("")+
  guides(fill=FALSE)
mdf<-as.data.frame(svm_m)
mdf<-cbind(mdf,id=seq(1:61))
mdf<-reshape2::melt(mdf,id.vars=c("id"))
h2<-ggplot(mdf)+
  geom_tile(aes(x=id,y=variable,fill=value),
            colour = "white") +
  scale_fill_gradient(low = "white",
    high = "orange")+ylab("")+xlab("")+
  guides(fill=FALSE)
mdf<-as.data.frame(rf_m)
mdf<-cbind(mdf,id=seq(1:61))
mdf<-reshape2::melt(mdf,id.vars=c("id"))
h3<-ggplot(mdf)+
  geom_tile(aes(x=id,y=variable,fill=value),
            colour = "white") +
  scale_fill_gradient(low = "white",
    high = "orange")+ylab("")+xlab("")+
  guides(fill=FALSE)
mdf<-as.data.frame(nb_m)
mdf<-cbind(mdf,id=seq(1:61))
mdf<-reshape2::melt(mdf,id.vars=c("id"))
h4<-ggplot(mdf)+
  geom_tile(aes(x=id,y=variable,fill=value),
            colour = "white") +
  scale_fill_gradient(low = "white",
    high = "orange")+ylab("")+xlab("")+
  guides(fill=FALSE)
grid.arrange(h1,h2,h3,h4,ncol=2,nrow=2)

difference heatmaps

mdf<-as.data.frame(rf_m - knn_m)
mdf<-cbind(mdf,id=seq(1:61))
mdf<-reshape2::melt(mdf,id.vars=c("id"))
mdf<-cbind(mdf,subclass=(botnet_prob_result[1:1830,]$subclass))
diff<-ggplot(mdf)+
  geom_tile(aes(x=id,y=variable,fill=value,text=subclass),
            colour = "white") +
  scale_fill_gradientn(colours=c("red","white","green"),
           values  = rescale(c(min(mdf$value), 0.05, max(mdf$value))))+
           guides(fill=FALSE)+theme_bw()  
Ignoring unknown aesthetics: text
ggplotly(diff)

d3heatmap(rf_m - knn_m,colors = "Blues",cellnote=matrix(botnet_prob_result[1:1830,]$subclass,ncol=30,nrow=61))

Subclass probability analisys (Attempt I)

KNN vs RF.

a=mdf %>% filter(value< -0.09) %>% group_by(subclass) %>% summarise(totless009=n()) %>% arrange(desc(totless009))
b=mdf  %>% group_by(subclass) %>% summarise(total=n()) %>% arrange(desc(total))
subclass_percent_diff<-inner_join(a,b,by="subclass") %>% mutate(percent=totless009/total) %>% arrange(desc(total)) 
subclass_percent_diff
subclass_detections<-botnet_prob_result %>% mutate(detected_rf=ifelse(RamdomForest>0.5,"Botnet","Normal"),
                              detected_knn=ifelse(KNN>0.5,"Botnet","Normal")) %>% 
                              mutate(correct_rf=ifelse(detected_rf==TrueClass,1,0))%>%
                              mutate(correct_knn=ifelse(detected_knn==TrueClass,1,0)) %>% 
  group_by(subclass) %>% summarise(total_correct_rf=sum(correct_rf),total_correct_knn=sum(correct_knn))
inner_join(subclass_detections,subclass_percent_diff,by="subclass")
mdf<-cbind(mdf,rf=botnet_prob_result$RamdomForest[1:1830],knn=botnet_prob_result$KNN[1:1830],trueclass=botnet_prob_result$TrueClass[1:1830])
botnet_prob_result
mdf %>% filter(value< -0.09) %>% filter(trueclass=="Botnet")
mdf %>% filter(value> 0.09) %>% filter(trueclass=="Normal")

Clustering and PCA

kmeans_mod<-kmeans(testing[,1:10],centers = 10)
testing_cluster<-cbind(testing,cluster=kmeans_mod$cluster)
pca<-prcomp(testing[,c(-11,-12)], center = TRUE, scale. = TRUE) 
pca_testing<-data.frame(pca$x,class=testing_cluster$class,
                              subclass=testing_cluster$subclass,
                               cluster=testing_cluster$cluster
                               )
g<-ggplot(pca_testing,aes(x=PC1,y=PC2))+
  geom_jitter(aes(color=as.factor(subclass),text=cluster,shape=class))+
  #geom_point(aes(shape=asignacion),size=3)+
  ylab("PC1")+xlab("PC2")+
  theme_classic()+
#scale_shape_manual(values=c(8,6))+
   guides(color=FALSE,alpha=FALSE)
Ignoring unknown aesthetics: text
ggplotly(g)
#Normal probabilistic table
normal_prob_result = data.frame(testing$class, predsrfprobs$Normal, knnPredict$Normal, logicRPredict$Normal, naiveBayesPredict$Normal ,svmPredict$Normal)
names(normal_prob_result) = c('True Class','Ramdom Forest','KNN','Logistic Regression', 'Naive Bayes', 'Suport VM')
normal_prob_result
LS0tCnRpdGxlOiAiQ29ubmVjdGlvbiBDbGFzc2lmaWNhdGlvbiIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKVGhpcyBpcyBhbiBbUiBNYXJrZG93bl0oaHR0cDovL3JtYXJrZG93bi5yc3R1ZGlvLmNvbSkgTm90ZWJvb2suIFdoZW4geW91IGV4ZWN1dGUgY29kZSB3aXRoaW4gdGhlIG5vdGVib29rLCB0aGUgcmVzdWx0cyBhcHBlYXIgYmVuZWF0aCB0aGUgY29kZS4gCgpUcnkgZXhlY3V0aW5nIHRoaXMgY2h1bmsgYnkgY2xpY2tpbmcgdGhlICpSdW4qIGJ1dHRvbiB3aXRoaW4gdGhlIGNodW5rIG9yIGJ5IHBsYWNpbmcgeW91ciBjdXJzb3IgaW5zaWRlIGl0IGFuZCBwcmVzc2luZyAqQ3RybCtTaGlmdCtFbnRlciouIAoKYGBge3J9CnN1cHByZXNzTWVzc2FnZXMobGlicmFyeSh0aWR5dmVyc2UpKQpzdXBwcmVzc01lc3NhZ2VzKGxpYnJhcnkoc3RyaW5ncikpCnN1cHByZXNzTWVzc2FnZXMobGlicmFyeShJU0xSKSkKc3VwcHJlc3NNZXNzYWdlcyhsaWJyYXJ5KGNhcmV0KSkKc3VwcHJlc3NNZXNzYWdlcyhsaWJyYXJ5KGRvTUMpKQpsaWJyYXJ5KHBsb3RseSkKcmVnaXN0ZXJEb01DKGNvcmVzPTQpCgpgYGAKCkFkZCBhIG5ldyBjaHVuayBieSBjbGlja2luZyB0aGUgKkluc2VydCBDaHVuayogYnV0dG9uIG9uIHRoZSB0b29sYmFyIG9yIGJ5IHByZXNzaW5nICpDdHJsK0FsdCtJKi4KCldoZW4geW91IHNhdmUgdGhlIG5vdGVib29rLCBhbiBIVE1MIGZpbGUgY29udGFpbmluZyB0aGUgY29kZSBhbmQgb3V0cHV0IHdpbGwgYmUgc2F2ZWQgYWxvbmdzaWRlIGl0IChjbGljayB0aGUgKlByZXZpZXcqIGJ1dHRvbiBvciBwcmVzcyAqQ3RybCtTaGlmdCtLKiB0byBwcmV2aWV3IHRoZSBIVE1MIGZpbGUpLgoKIyMjIEdldHRpbmcgYW5kIHByb2NjZXNpbmcgdGhlIGRhdGEKYGBge3J9CmxpYnJhcnkoc3RyaW5ncikKbXlEYXRhID0gcmVhZC5jc3YoJy9ob21lL2hhcnBvL0Ryb3Bib3gvb25nb2luZy13b3JrL2dpdC1yZXBvcy9sYWJlbGluZy1kYXRhc2V0cy9waGQvZGF0YXNldHMvZGF0YV9hbGxfcmVzdWx0LnR4dCcsIHN0cmluZ3NBc0ZhY3RvcnMgPSBGLCBzZXAgPSAnICcpCgojQ3JlYXRlIGRhdGEgYmFja3VwCm15RGF0YS5ia3VwIDwtIG15RGF0YQojQ3JlYXRlIG5ldyBjb2x1bW46IGxlbmd0aCBvZiBtb2RlbCwgYW5kIG51bWJlciBvZiBwZXJpb2RpY2l0eSwgZHVyYXRpb24gYW5kIHNpemUgY2hhcmFjdGVyaXN0aWMgaW4gdGhlIG1vZGVsLgpteURhdGEgPSBteURhdGEgJT4lIG11dGF0ZShsZXR0ZXJfY291bnQgPSBuY2hhcihTdGF0ZSkpCiNQZXJpb2RpY2l0eQpteURhdGEgPSBteURhdGEgJT4lIG11dGF0ZShzdHJvbmdfcCA9IHN0cl9jb3VudChTdGF0ZSwnW2EtaV0nKSkKbXlEYXRhID0gbXlEYXRhICU+JSBtdXRhdGUod2Vha19wID0gc3RyX2NvdW50KFN0YXRlLCdbQS1JXScpKQpteURhdGEgPSBteURhdGEgJT4lIG11dGF0ZSh3ZWFrX25wID0gc3RyX2NvdW50KFN0YXRlLCdbci16XScpKQpteURhdGEgPSBteURhdGEgJT4lIG11dGF0ZShzdHJvbmdfbnAgPSBzdHJfY291bnQoU3RhdGUsJ1tSLVpdJykpCiNEdXJhdGlvbgpteURhdGEgPSBteURhdGEgJT4lIG11dGF0ZShkdXJhdGlvbl9zID0gc3RyX2NvdW50KFN0YXRlLCcoYXxBfHJ8UnwxfGR8RHx1fFV8NHxnfEd8eHxYfDcpJykpI2MoJ2EnLCdBJywncicsJ1InLCcxJywnZCcsJ0QnLCd1JywnVScsJzQnLCdnJywnRycsJ3gnLCdYJywnNycpKSkKbXlEYXRhID0gbXlEYXRhICU+JSBtdXRhdGUoZHVyYXRpb25fbSA9IHN0cl9jb3VudChTdGF0ZSwnKGJ8QnxzfFN8MnxlfEV8dnxWfDV8aHxIfHl8WXw4KScpKSNjKCdiJywnQicsJ3MnLCdTJywnMicsJ2UnLCdFJywndicsJ1YnLCc1JywnaCcsJ0gnLCd5JywnWScsJzgnKSkpCm15RGF0YSA9IG15RGF0YSAlPiUgbXV0YXRlKGR1cmF0aW9uX2wgPSBzdHJfY291bnQoU3RhdGUsJyhjfEN8dHxUfDN8ZnxGfHd8V3w2fGl8SXx6fFp8OSknKSkjYygnYycsJ0MnLCd0JywnVCcsJzMnLCdmJywnRicsJ3cnLCdXJywnNicsJ2knLCdJJywneicsJ1onLCc5JykpKQojU2l6ZQpteURhdGEgPSBteURhdGEgJT4lIG11dGF0ZShzaXplX3MgPSBzdHJfY291bnQoU3RhdGUsJ1thLWNdJykgKyBzdHJfY291bnQoU3RhdGUsJ1tBLUNdJykgKyBzdHJfY291bnQoU3RhdGUsJ1tyLXRdJykgKyBzdHJfY291bnQoU3RhdGUsJ1tSLVRdJykgKyBzdHJfY291bnQoU3RhdGUsJ1sxLTNdJykpCm15RGF0YSA9IG15RGF0YSAlPiUgbXV0YXRlKHNpemVfbSA9IHN0cl9jb3VudChTdGF0ZSwnW2QtZl0nKSArIHN0cl9jb3VudChTdGF0ZSwnW0QtRl0nKSArIHN0cl9jb3VudChTdGF0ZSwnW3Utd10nKSArIHN0cl9jb3VudChTdGF0ZSwnW1UtV10nKSArIHN0cl9jb3VudChTdGF0ZSwnWzQtNl0nKSkKbXlEYXRhID0gbXlEYXRhICU+JSBtdXRhdGUoc2l6ZV9sID0gc3RyX2NvdW50KFN0YXRlLCdbZy1pXScpICsgc3RyX2NvdW50KFN0YXRlLCdbRy1JXScpICsgc3RyX2NvdW50KFN0YXRlLCdbeC16XScpICsgc3RyX2NvdW50KFN0YXRlLCdbWC1aXScpICsgc3RyX2NvdW50KFN0YXRlLCdbNy05XScpKQoKI1JlbW92ZSBmcm9tIExhYmVsTmFtZSB1bm5lY2Vzc2FyeSBjaGFyYWN0ZXJzIChlajogVjQyLCAtMTcpCm15RGF0YSA8LSBteURhdGEgJT4lIG11dGF0ZShMYWJlbE5hbWUgPSBnc3ViKCdWWzAtOV0rLScsJycsTGFiZWxOYW1lKSkKbXlEYXRhIDwtIG15RGF0YSAlPiUgbXV0YXRlKExhYmVsTmFtZSA9IGdzdWIoJy1bMC05XSsnLCcnLExhYmVsTmFtZSkpCm15RGF0YSA8LSBteURhdGEgJT4lIG11dGF0ZShMYWJlbE5hbWUgPSBnc3ViKCdDQ1swLTldKy0nLCdDQy0nLExhYmVsTmFtZSkpCgojS2VlcCBvbmx5IGNvbm5lY3Rpb24gd2l0aCBtb3JlIHRoYW4gMyBzeW1ib2xzCm15RGF0YSA8LSBteURhdGEgJT4lIGZpbHRlcihsZXR0ZXJfY291bnQgPiAzKQoKI1BlcmlvZGljaXR5ICUKbXlEYXRhIDwtIG15RGF0YSAlPiUgbXV0YXRlKHN0cm9uZ19wID0gKHN0cm9uZ19wIC8gbGV0dGVyX2NvdW50KSkKbXlEYXRhIDwtIG15RGF0YSAlPiUgbXV0YXRlKHdlYWtfcCA9ICh3ZWFrX3AgLyBsZXR0ZXJfY291bnQpKQpteURhdGEgPC0gbXlEYXRhICU+JSBtdXRhdGUoc3Ryb25nX25wID0gKHN0cm9uZ19ucCAvIGxldHRlcl9jb3VudCkpCm15RGF0YSA8LSBteURhdGEgJT4lIG11dGF0ZSh3ZWFrX25wID0gKHdlYWtfbnAgLyBsZXR0ZXJfY291bnQpKQojRHVyYXRpb24gJQpteURhdGEgPC0gbXlEYXRhICU+JSBtdXRhdGUoZHVyYXRpb25fcyA9IChkdXJhdGlvbl9zIC8gbGV0dGVyX2NvdW50KSkKbXlEYXRhIDwtIG15RGF0YSAlPiUgbXV0YXRlKGR1cmF0aW9uX20gPSAoZHVyYXRpb25fbSAvIGxldHRlcl9jb3VudCkpCm15RGF0YSA8LSBteURhdGEgJT4lIG11dGF0ZShkdXJhdGlvbl9sID0gKGR1cmF0aW9uX2wgLyBsZXR0ZXJfY291bnQpKQojU2l6ZSAlCm15RGF0YSA8LSBteURhdGEgJT4lIG11dGF0ZShzaXplX3MgPSAoc2l6ZV9zIC8gbGV0dGVyX2NvdW50KSkKbXlEYXRhIDwtIG15RGF0YSAlPiUgbXV0YXRlKHNpemVfbSA9IChzaXplX20gLyBsZXR0ZXJfY291bnQpKQpteURhdGEgPC0gbXlEYXRhICU+JSBtdXRhdGUoc2l6ZV9sID0gKHNpemVfbCAvIGxldHRlcl9jb3VudCkpCgojaGVhZChteURhdGEpCm15RGF0YVsxOjIwLF0KCiNNYWtpbmcgZmVhdHVyZSB2ZWN0b3JzCmZlYXR1cmVfdmVjdG9ycyA9IG15RGF0YVssYygnc3Ryb25nX3AnLCd3ZWFrX3AnLCd3ZWFrX25wJywnc3Ryb25nX25wJywnZHVyYXRpb25fcycsJ2R1cmF0aW9uX20nLCdkdXJhdGlvbl9sJywnc2l6ZV9zJywnc2l6ZV9tJywnc2l6ZV9sJywnbGV0dGVyX2NvdW50JywnTGFiZWwnLCdMYWJlbE5hbWUnKV0KbmFtZXMoZmVhdHVyZV92ZWN0b3JzKSA9IGMoInNwIiwid3AiLCJ3bnAiLCJzbnAiLCJkcyIsImRtIiwiZGwiLCJzcyIsInNtIiwic2wiLCJsZW5ndGgiLCJjbGFzcyIsInN1YmNsYXNzIikKZmVhdHVyZV92ZWN0b3JzJGNsYXNzID0gZmFjdG9yKGZlYXR1cmVfdmVjdG9ycyRjbGFzcykKZmVhdHVyZV92ZWN0b3JzJHN1YmNsYXNzID0gZmFjdG9yKGZlYXR1cmVfdmVjdG9ycyRzdWJjbGFzcykKYGBgCgojIyMgQ3JlYXRlIHRyYWluaW5nIHNldCBhbmQgdGVzdHNldApgYGB7cn0Kc2V0LnNlZWQoMzAwKQp0cmFpbkluZGV4IDwtIGNyZWF0ZURhdGFQYXJ0aXRpb24oZmVhdHVyZV92ZWN0b3JzJGNsYXNzLCBwPTAuODAsIGxpc3Q9RkFMU0UpCmRhdGFfdHJhaW4gPC0gZmVhdHVyZV92ZWN0b3JzWyB0cmFpbkluZGV4LF0KZGF0YV90ZXN0IDwtIGZlYXR1cmVfdmVjdG9yc1stdHJhaW5JbmRleCxdCgojZGF0YV90cmFpbiA9IGRhdGFfdHJhaW4gJT4lIGZpbHRlcihsZW5ndGg+NSkKdHJhaW4gPC0gdXBTYW1wbGUoeCA9IGRhdGFfdHJhaW4sICB5ID0gZGF0YV90cmFpbiRjbGFzcywgeW5hbWU9ImNsYXNzIikKI3RyYWluIDwtIHVwU2FtcGxlKHggPSB0cmFpbiwgIHkgPSB0cmFpbiRzdWJjbGFzcywgeW5hbWU9ImNsYXNzIikKdHJhaW5pbmcgPC0gdHJhaW5bLC1jKDExLDEyKV0KdGVzdGluZyA8LSBkYXRhX3Rlc3RbLC1jKDExKV0KdHJhaW5pbmcKdHJhaW4KY3RybF9mYXN0IDwtIHRyYWluQ29udHJvbChtZXRob2Q9ImN2IiwgCiAgICAgICAgICAgICAgICAgICAgIHJlcGVhdHM9MiwKICAgICAgICAgICAgICAgICAgICAgbnVtYmVyPTEwLCAKICAgICAgICAgICAgICAgICAgICAgc3VtbWFyeUZ1bmN0aW9uPXR3b0NsYXNzU3VtbWFyeSwKICAgICAgICAgICAgICAgICAgICAgdmVyYm9zZUl0ZXI9VCwKICAgICAgICAgICAgICAgICAgICAgY2xhc3NQcm9icz1UUlVFLAogICAgICAgICAgICAgICAgICAgICBhbGxvd1BhcmFsbGVsID0gVFJVRSkgIApjdHJsIDwtIHRyYWluQ29udHJvbChtZXRob2Q9InJlcGVhdGVkY3YiLHJlcGVhdHMgPSAzKSAjLGNsYXNzUHJvYnM9VFJVRSxzdW1tYXJ5RnVuY3Rpb24gPSB0d29DbGFzc1N1bW1hcnkpCmBgYAoKCiMjIyBSYW5kb20gRm9yZXN0IENsYXNzaWZpY2F0b3IKYGBge3J9CiAgIyBSYW5kb20gRm9yZXN0CnJmRml0IDwtIHRyYWluKGNsYXNzIH4gLiwKICAgICAgICAgICAgICAgZGF0YSA9IHRyYWluaW5nLAogICAgICAgICAgICAgICBtZXRyaWM9IlJPQyIsCiAgICAgICAgICAgICAgIG1ldGhvZCA9ICJyZiIsCiAgICAgICAgICAgICAgIHRyQ29udHJvbCA9IGN0cmxfZmFzdCkKCnJmRml0CnJmRml0JGZpbmFsTW9kZWwKYGBgCgpgYGB7cn0KcHJlZHNyZnByb2JzPXByZWRpY3QocmZGaXQsdGVzdGluZyx0eXBlPSdwcm9iJykKcHJlZHNyZj1pZmVsc2UocHJlZHNyZnByb2JzJEJvdG5ldCA+PTAuOSwnQm90bmV0JywnTm9ybWFsJykKY29uZnVzaW9uTWF0cml4KHByZWRzcmYsdGVzdGluZyRjbGFzcykKYGBgCgpgYGB7cn0KbGlicmFyeShnZ3Bsb3QyKQpsaWJyYXJ5KHBsb3RST0MpCnNlbGVjdGVkSW5kaWNlcyA8LSByZkZpdCRwcmVkJG10cnkgPT0gMgpnZ3Bsb3QoY2JpbmQocHJlZHNyZnByb2JzLGNsYXNzPXRlc3RpbmckY2xhc3MpLCAKICAgICAgIGFlcyhtID0gQm90bmV0LCBkID0gZmFjdG9yKGNsYXNzLCBsYWJlbHM9YygiTm9ybWFsIiwiQm90bmV0IiksbGV2ZWxzID0gYygiTm9ybWFsIiwgIkJvdG5ldCIpKSkpICsgCiAgICBnZW9tX3JvYyhoanVzdCA9IC0wLjQsIHZqdXN0ID0gMS41LGNvbG91cj0nb3JhbmdlJykgKyAKICB0aGVtZV9idygpCgpjYmluZChwcmVkc3JmcHJvYnMsY2xhc3M9dGVzdGluZyRjbGFzcykKYGBgCgoKIyMjIEtOTgpgYGB7cn0KI0NoZWNraW5nIGRpc3RpYnV0aW9uIGluIG9yaWdhbmwgZGF0YSBhbmQgcGFydGl0aW9uZWQgZGF0YQpwcm9wLnRhYmxlKHRhYmxlKHRyYWluaW5nJGNsYXNzKSkgKiAxMDAKcHJvcC50YWJsZSh0YWJsZSh0ZXN0aW5nJGNsYXNzKSkgKiAxMDAKcHJvcC50YWJsZSh0YWJsZShmZWF0dXJlX3ZlY3RvcnMkY2xhc3MpKSAqIDEwMAoKdHJhaW5YIDwtIHRyYWluaW5nWyxuYW1lcyh0cmFpbmluZykgIT0gImNsYXNzIl0KcHJlUHJvY1ZhbHVlcyA8LSBwcmVQcm9jZXNzKHggPSB0cmFpblgsbWV0aG9kID0gYygiY2VudGVyIiwgInNjYWxlIikpCnByZVByb2NWYWx1ZXMKYGBgCmBgYHtyfQprbm5GaXQgPC0gdHJhaW4oY2xhc3MgfiAuLCBkYXRhID0gdHJhaW5pbmcsIG1ldGhvZCA9ICJrbm4iLCB0ckNvbnRyb2wgPSBjdHJsX2Zhc3QsIHByZVByb2Nlc3MgPSBjKCJjZW50ZXIiLCJzY2FsZSIpLCB0dW5lTGVuZ3RoID0gMjApCgojT3V0cHV0IG9mIGtOTiBmaXQKa25uRml0CmBgYApgYGB7cn0KI1Bsb3R0aW5nIHlpZWxkcyBOdW1iZXIgb2YgTmVpZ2hib3VycyBWcyBhY2N1cmFjeSAoYmFzZWQgb24gcmVwZWF0ZWQgY3Jvc3MgdmFsaWRhdGlvbikKcGxvdChrbm5GaXQpCmBgYApgYGB7cn0Ka25uUHJlZGljdCA8LSBwcmVkaWN0KGtubkZpdCxuZXdkYXRhID0gdGVzdGluZyApCiNHZXQgdGhlIGNvbmZ1c2lvbiBtYXRyaXggdG8gc2VlIGFjY3VyYWN5IHZhbHVlIGFuZCBvdGhlciBwYXJhbWV0ZXIgdmFsdWVzCmNvbmZ1c2lvbk1hdHJpeChrbm5QcmVkaWN0LCB0ZXN0aW5nJGNsYXNzICkKYGBgCmBgYHtyfQptZWFuKGtublByZWRpY3QgPT0gdGVzdGluZyRjbGFzcykKYGBgCmBgYHtyfQpsaWJyYXJ5KHBST0MpCmtublByZWRpY3QgPC0gcHJlZGljdChrbm5GaXQsbmV3ZGF0YSA9IHRlc3RpbmcgLCB0eXBlPSJwcm9iIikKa25uUk9DIDwtIHJvYyh0ZXN0aW5nJGNsYXNzLGtublByZWRpY3RbLCJCb3RuZXQiXSwgbGV2ZWxzID0gYygnTm9ybWFsJywnQm90bmV0JykpI3Jldih0ZXN0aW5nJGNsYXNzKSkKa25uUk9DCmBgYAoKYGBge3J9CmdncGxvdChjYmluZChrbm5QcmVkaWN0LGNsYXNzPXRlc3RpbmckY2xhc3MpLCAKICAgICAgIGFlcyhtID0gQm90bmV0LCBkID0gZmFjdG9yKGNsYXNzLCBsYWJlbHM9YygiTm9ybWFsIiwiQm90bmV0IiksbGV2ZWxzID0gYygiTm9ybWFsIiwgIkJvdG5ldCIpKSkpICsgCiAgICBnZW9tX3JvYyhoanVzdCA9IC0wLjQsIHZqdXN0ID0gMS41LGNvbG91cj0nb3JhbmdlJykgKyAKICB0aGVtZV9idygpCgojcGxvdChrbm5ST0MsIHR5cGU9IlMiLCBwcmludC50aHJlcz0gMC41KQpgYGAKIyMjIExvZ2lzdGljIFJlZ3Jlc3Npb24KCmBgYHtyfQpsb2dpY1JGaXQgPC0gdHJhaW4oY2xhc3MgfiAuLCBtZXRob2Q9J2dsbScsIHRyQ29udHJvbCA9IGN0cmxfZmFzdCxwcmVQcm9jZXNzPWMoJ3NjYWxlJywgJ2NlbnRlcicpLCBkYXRhPXRyYWluaW5nLCBmYW1pbHk9Ymlub21pYWwobGluaz0nbG9naXQnKSkKI2xvZ2ljUkZpdCA8LSB0cmFpbihjbGFzcyB+IHNwKndwKnducCpzbnAqZHMqZG0qZGwqc3Mqc20qc2wsIG1ldGhvZD0nZ2xtJywgdHJDb250cm9sID0gY3RybF9mYXN0LHByZVByb2Nlc3M9Yygnc2NhbGUnLCAnY2VudGVyJyksIGRhdGE9dHJhaW5pbmcsIGZhbWlseT1iaW5vbWlhbChsaW5rPSdsb2dpdCcpKQojbG9naWNSRml0IDwtIHRyYWluKGNsYXNzIH4gc3Ard3Ard25wK3NucCtkcytkbStkbCtzcytzbStzbCwgbWV0aG9kPSdnbG0nLCB0ckNvbnRyb2wgPSBjdHJsX2Zhc3QscHJlUHJvY2Vzcz1jKCdzY2FsZScsICdjZW50ZXInKSwgZGF0YT10cmFpbmluZywgZmFtaWx5PWJpbm9taWFsKGxpbms9J2xvZ2l0JykpCgojc3VtbWFyeShsb2dpY1JGaXQpCiNPdXRwdXQgb2YgTG9naXN0aWMgUmVncmVzc2lvbiBmaXQKbG9naWNSRml0CmBgYAoKYGBge3J9Cgpsb2dpY1JQcmVkaWN0IDwtIHByZWRpY3QobG9naWNSRml0LCBuZXdkYXRhID0gdGVzdGluZyApCgpjb25mdXNpb25NYXRyaXgobG9naWNSUHJlZGljdCwgdGVzdGluZyRjbGFzcykKYGBgCmBgYHtyfQpsb2dpY1JQcmVkaWN0IDwtIHByZWRpY3QobG9naWNSRml0LCBuZXdkYXRhID0gdGVzdGluZywgdHlwZT0icHJvYiIpCmxvZ2ljUk9DIDwtIHJvYyh0ZXN0aW5nJGNsYXNzLGxvZ2ljUlByZWRpY3RbLCJCb3RuZXQiXSwgbGV2ZWxzID0gYygnTm9ybWFsJywnQm90bmV0JykpI3Jldih0ZXN0aW5nJGNsYXNzKSkKCmdncGxvdChjYmluZChsb2dpY1JQcmVkaWN0LGNsYXNzPXRlc3RpbmckY2xhc3MpLCAKICAgICAgIGFlcyhtID0gQm90bmV0LCBkID0gZmFjdG9yKGNsYXNzLCBsYWJlbHM9YygiTm9ybWFsIiwiQm90bmV0IiksbGV2ZWxzID0gYygiTm9ybWFsIiwgIkJvdG5ldCIpKSkpICsgCiAgICBnZW9tX3JvYyhoanVzdCA9IC0wLjQsIHZqdXN0ID0gMS41LGNvbG91cj0nb3JhbmdlJykgKyAKICB0aGVtZV9idygpCgojbG9naWNST0MKYGBgCgojIyMgTmFpdmUgQmF5ZXMKYGBge3J9Cm5haXZlQmF5ZXNGaXQgPC0gdHJhaW4oY2xhc3MgfiAuLCBtZXRob2Q9J25iJywgdHJDb250cm9sID0gY3RybF9mYXN0LHByZVByb2Nlc3M9Yygnc2NhbGUnLCAnY2VudGVyJyksIGRhdGE9dHJhaW5pbmcpCm5haXZlQmF5ZXNGaXQKYGBgCgpgYGB7cn0KbmFpdmVCYXllc1ByZWRpY3QgPC0gcHJlZGljdChuYWl2ZUJheWVzRml0LCBuZXdkYXRhID0gdGVzdGluZykKCmNvbmZ1c2lvbk1hdHJpeChuYWl2ZUJheWVzUHJlZGljdCwgdGVzdGluZyRjbGFzcykKYGBgCmBgYHtyfQpuYWl2ZUJheWVzUHJlZGljdCA8LSBwcmVkaWN0KG5haXZlQmF5ZXNGaXQsIG5ld2RhdGEgPSB0ZXN0aW5nLCB0eXBlID0gJ3Byb2InKQpuYWl2ZUJheWVzUk9DIDwtIHJvYyh0ZXN0aW5nJGNsYXNzLG5haXZlQmF5ZXNQcmVkaWN0WywiQm90bmV0Il0sIGxldmVscyA9IGMoJ05vcm1hbCcsJ0JvdG5ldCcpKSNyZXYodGVzdGluZyRjbGFzcykpCm5haXZlQmF5ZXNST0MKCmdncGxvdChjYmluZChuYWl2ZUJheWVzUHJlZGljdCxjbGFzcz10ZXN0aW5nJGNsYXNzKSwgCiAgICAgICBhZXMobSA9IEJvdG5ldCwgZCA9IGZhY3RvcihjbGFzcywgbGFiZWxzPWMoIk5vcm1hbCIsIkJvdG5ldCIpLGxldmVscyA9IGMoIk5vcm1hbCIsICJCb3RuZXQiKSkpKSArIAogICAgZ2VvbV9yb2MoaGp1c3QgPSAtMC40LCB2anVzdCA9IDEuNSxjb2xvdXI9J29yYW5nZScpICsgCiAgdGhlbWVfYncoKQoKI3Bsb3QobmFpdmVCYXllc1JPQywgdHlwZT0iUyIsIHByaW50LnRocmVzPSAwLjUpCmBgYAoKCiMjIyBTdXBvcnQgVmVjdG9yIE1hY2hpbmUKYGBge3J9CnN2bUZpdCA8LSB0cmFpbihjbGFzcyB+IC4sIG1ldGhvZD0nc3ZtTGluZWFyJywgdHJDb250cm9sID0gY3RybF9mYXN0LHByZVByb2Nlc3M9Yygnc2NhbGUnLCAnY2VudGVyJyksIGRhdGE9dHJhaW5pbmcsIGZhbWlseT1iaW5vbWlhbChsaW5rPSdsb2dpdCcpKQpzdm1GaXQKYGBgCgpgYGB7cn0Kc3ZtUHJlZGljdCA8LSBwcmVkaWN0KHN2bUZpdCwgbmV3ZGF0YSA9IHRlc3RpbmcpCmNvbmZ1c2lvbk1hdHJpeChzdm1QcmVkaWN0LCB0ZXN0aW5nJGNsYXNzKQpgYGAKCmBgYHtyfQpzdm1QcmVkaWN0IDwtIHByZWRpY3Qoc3ZtRml0LCBuZXdkYXRhID0gdGVzdGluZywgdHlwZSA9ICJwcm9iIikKCnN2bVJPQyA8LSByb2ModGVzdGluZyRjbGFzcyxzdm1QcmVkaWN0WywiQm90bmV0Il0sIGxldmVscyA9IGMoJ05vcm1hbCcsJ0JvdG5ldCcpKSNyZXYodGVzdGluZyRjbGFzcykpCnN2bVJPQwoKZ2dwbG90KGNiaW5kKHN2bVByZWRpY3QsY2xhc3M9dGVzdGluZyRjbGFzcyksIAogICAgICAgYWVzKG0gPSBCb3RuZXQsIGQgPSBmYWN0b3IoY2xhc3MsIGxhYmVscz1jKCJOb3JtYWwiLCJCb3RuZXQiKSxsZXZlbHMgPSBjKCJOb3JtYWwiLCAiQm90bmV0IikpKSkgKyAKICAgIGdlb21fcm9jKGhqdXN0ID0gLTAuNCwgdmp1c3QgPSAxLjUsY29sb3VyPSdvcmFuZ2UnKSArIAogIHRoZW1lX2J3KCkKCmBgYAoKCiMjIyBDb21wYXJpbmcgTW9kZWxzCmBgYHtyfQpyZXNhbXBzIDwtIHJlc2FtcGxlcyhsaXN0KHJmID0gcmZGaXQsIGxyID0gbG9naWNSRml0LCBudiA9IG5haXZlQmF5ZXNGaXQsIHN2bSA9IHN2bUZpdCkpCnN1bW1hcnkocmVzYW1wcykKYndwbG90KHJlc2FtcHMpCmRpZmZzIDwtIGRpZmYocmVzYW1wcykKc3VtbWFyeShkaWZmcykKdmFsdWVzPXJlc2FtcHMkdmFsdWVzCnZhbHVlcwpuYW1lcyh2YWx1ZXMpWzJdPC0icmZTZW5zIgoKZ2dwbG90KHZhbHVlcykrCiAgZ2VvbV9ib3hwbG90KGFlcyh5PXJmU2Vucyx4PTEpKQoKYGBgCiMjIyBNYWtpbmcgcHJvYmFiaWxpc3RpYyB0YWJsZS4KYGBge3J9CiMgQm90bmV0IHByb2JhYmlsaXN0aWMgdGFibGUKYm90bmV0X3Byb2JfcmVzdWx0ID0gZGF0YS5mcmFtZSh0ZXN0aW5nJGNsYXNzLCBwcmVkc3JmcHJvYnMkQm90bmV0LCBrbm5QcmVkaWN0JEJvdG5ldCwgbG9naWNSUHJlZGljdCRCb3RuZXQsIG5haXZlQmF5ZXNQcmVkaWN0JEJvdG5ldCAsc3ZtUHJlZGljdCRCb3RuZXQpCmJvdG5ldF9wcm9iX3Jlc3VsdCA9IGJvdG5ldF9wcm9iX3Jlc3VsdCAlPiUgbXV0YXRlKHN1YmNsYXNzID0gZGF0YV90ZXN0JHN1YmNsYXNzKQpuYW1lcyhib3RuZXRfcHJvYl9yZXN1bHQpID0gYygnVHJ1ZUNsYXNzJywnUmFtZG9tRm9yZXN0JywnS05OJywnTG9naXN0aWNSZWdyZXNzaW9uJywgJ05haXZlQmF5ZXMnLCAnU1ZNJywnc3ViY2xhc3MnKQpib3RuZXRfcHJvYl9yZXN1bHQKYGBgCgoKYGBge3J9CiNsb2FkKCIuL2JvdG5ldF9wcm9iX3Jlc3VsdHMuUmRhIikKbGlicmFyeShncmlkKQpsaWJyYXJ5KGdyaWRFeHRyYSkKI2JvdG5ldF9wcm9iX3Jlc3VsdCAlPiUgZ3JvdXBfYnkoc3ViY2xhc3MpICU+JSBzdW1tYXJpc2Uobj1uKCksbWVhbj1tZWFuKEtOTiksc2Q9c2QoS05OKSkgJT4lIGFycmFuZ2UoZGVzYyhuKSkKI2JvdG5ldF9wcm9iX3Jlc3VsdCAlPiUgZ3JvdXBfYnkoc3ViY2xhc3MpICU+JSBzdW1tYXJpc2Uobj1uKCksbWVhbj1tZWFuKE5haXZlQmF5ZXMpLHNkPXNkKE5haXZlQmF5ZXMpKSAlPiUgYXJyYW5nZShkZXNjKG4pKSU+JSB0b3BfbigxMCkKCmJvdG5ldF8xMF90b3A8LWJvdG5ldF9wcm9iX3Jlc3VsdCAlPiUgZmlsdGVyKFRydWVDbGFzcz09Ik5vcm1hbCIpICU+JSBncm91cF9ieShzdWJjbGFzcykgJT4lIHN1bW1hcmlzZShuPW4oKSkgJT4lIGFycmFuZ2UoZGVzYyhuKSklPiUgdG9wX24oMjApCmJvdG5ldF8xMF90b3A8LWlubmVyX2pvaW4oYm90bmV0XzEwX3RvcCxib3RuZXRfcHJvYl9yZXN1bHQsYnk9InN1YmNsYXNzIikKCmtubl9wbG90PC1id3Bsb3Qoc3ViY2xhc3N+S05OLGRhdGE9Ym90bmV0XzEwX3RvcCxkby5vdXQgPSBGQUxTRSxzY2FsZXM9bGlzdCh5PWxpc3QoZHJhdz1GQUxTRSkpKQpyZl9wbG90PC1id3Bsb3Qoc3ViY2xhc3N+UmFtZG9tRm9yZXN0LGRhdGE9Ym90bmV0XzEwX3RvcCxkby5vdXQgPSBGQUxTRSxzY2FsZXM9bGlzdCh5PWxpc3QoZHJhdz1GQUxTRSkpKQpybF9wbG90PC1id3Bsb3Qoc3ViY2xhc3N+TG9naXN0aWNSZWdyZXNzaW9uLGRhdGE9Ym90bmV0XzEwX3RvcCxkby5vdXQgPSBGQUxTRSxzY2FsZXM9bGlzdCh5PWxpc3QoZHJhdz1GQUxTRSkpKQpzdm1fcGxvdDwtYndwbG90KHN1YmNsYXNzflNWTSxkYXRhPWJvdG5ldF8xMF90b3AsZG8ub3V0ID0gRkFMU0Usc2NhbGVzPWxpc3QoeT1saXN0KGRyYXc9RkFMU0UpKSkKCgpwbCA9IGxpc3Qoa25uX3Bsb3QsIHJmX3Bsb3QscmxfcGxvdCxzdm1fcGxvdCkKIyBkby5jYWxsKGdyaWQuYXJyYW5nZSwgYyhwbCwgbnJvdz0xKSkKZG8uY2FsbChncmlkLmFycmFuZ2UsIGMobGFwcGx5KHBsLCB1cGRhdGUpLCBsaXN0KG5yb3c9MSkpKQoKYGBgCgojIyMjIHNvbWUgaGVhdG1hcHMKYGBge3IgaGVhdG1hcHN9CmxpYnJhcnkoc2NhbGVzKQprbm5fbTwtbWF0cml4KGJvdG5ldF9wcm9iX3Jlc3VsdFsxOjE4MzAsXSRLTk4sbmNvbD0zMCxucm93PTYxKQpzdm1fbTwtbWF0cml4KGJvdG5ldF9wcm9iX3Jlc3VsdFsxOjE4MzAsXSRTVk0sbmNvbD0zMCxucm93PTYxKQpscl9tPC1tYXRyaXgoYm90bmV0X3Byb2JfcmVzdWx0WzE6MTgzMCxdJExvZ2lzdGljUmVncmVzc2lvbixuY29sPTMwLG5yb3c9NjEpCm5iX208LW1hdHJpeChib3RuZXRfcHJvYl9yZXN1bHRbMToxODMwLF0kTmFpdmVCYXllcyxuY29sPTMwLG5yb3c9NjEpCnJmX208LW1hdHJpeChib3RuZXRfcHJvYl9yZXN1bHRbMToxODMwLF0kUmFtZG9tRm9yZXN0LG5jb2w9MzAsbnJvdz02MSkKCm1kZjwtYXMuZGF0YS5mcmFtZShrbm5fbSkKbWRmPC1jYmluZChtZGYsaWQ9c2VxKDE6NjEpKQptZGY8LXJlc2hhcGUyOjptZWx0KG1kZixpZC52YXJzPWMoImlkIikpCmgxPC1nZ3Bsb3QobWRmKSsKICBnZW9tX3RpbGUoYWVzKHg9aWQseT12YXJpYWJsZSxmaWxsPXZhbHVlKSwKICAgICAgICAgICAgY29sb3VyID0gIndoaXRlIikgKwogIHNjYWxlX2ZpbGxfZ3JhZGllbnQobG93ID0gIndoaXRlIiwKICAgIGhpZ2ggPSAib3JhbmdlIikreWxhYigiIikreGxhYigiIikrCiAgZ3VpZGVzKGZpbGw9RkFMU0UpCgoKbWRmPC1hcy5kYXRhLmZyYW1lKHN2bV9tKQptZGY8LWNiaW5kKG1kZixpZD1zZXEoMTo2MSkpCm1kZjwtcmVzaGFwZTI6Om1lbHQobWRmLGlkLnZhcnM9YygiaWQiKSkKaDI8LWdncGxvdChtZGYpKwogIGdlb21fdGlsZShhZXMoeD1pZCx5PXZhcmlhYmxlLGZpbGw9dmFsdWUpLAogICAgICAgICAgICBjb2xvdXIgPSAid2hpdGUiKSArCiAgc2NhbGVfZmlsbF9ncmFkaWVudChsb3cgPSAid2hpdGUiLAogICAgaGlnaCA9ICJvcmFuZ2UiKSt5bGFiKCIiKSt4bGFiKCIiKSsKICBndWlkZXMoZmlsbD1GQUxTRSkKCgptZGY8LWFzLmRhdGEuZnJhbWUocmZfbSkKbWRmPC1jYmluZChtZGYsaWQ9c2VxKDE6NjEpKQptZGY8LXJlc2hhcGUyOjptZWx0KG1kZixpZC52YXJzPWMoImlkIikpCmgzPC1nZ3Bsb3QobWRmKSsKICBnZW9tX3RpbGUoYWVzKHg9aWQseT12YXJpYWJsZSxmaWxsPXZhbHVlKSwKICAgICAgICAgICAgY29sb3VyID0gIndoaXRlIikgKwogIHNjYWxlX2ZpbGxfZ3JhZGllbnQobG93ID0gIndoaXRlIiwKICAgIGhpZ2ggPSAib3JhbmdlIikreWxhYigiIikreGxhYigiIikrCiAgZ3VpZGVzKGZpbGw9RkFMU0UpCgoKbWRmPC1hcy5kYXRhLmZyYW1lKG5iX20pCm1kZjwtY2JpbmQobWRmLGlkPXNlcSgxOjYxKSkKbWRmPC1yZXNoYXBlMjo6bWVsdChtZGYsaWQudmFycz1jKCJpZCIpKQpoNDwtZ2dwbG90KG1kZikrCiAgZ2VvbV90aWxlKGFlcyh4PWlkLHk9dmFyaWFibGUsZmlsbD12YWx1ZSksCiAgICAgICAgICAgIGNvbG91ciA9ICJ3aGl0ZSIpICsKICBzY2FsZV9maWxsX2dyYWRpZW50KGxvdyA9ICJ3aGl0ZSIsCiAgICBoaWdoID0gIm9yYW5nZSIpK3lsYWIoIiIpK3hsYWIoIiIpKwogIGd1aWRlcyhmaWxsPUZBTFNFKQpncmlkLmFycmFuZ2UoaDEsaDIsaDMsaDQsbmNvbD0yLG5yb3c9MikKYGBgCgojIyMjIGRpZmZlcmVuY2UgaGVhdG1hcHMKYGBge3IgaGVhdG1hcCBkaWZmfQptZGY8LWFzLmRhdGEuZnJhbWUocmZfbSAtIGtubl9tKQptZGY8LWNiaW5kKG1kZixpZD1zZXEoMTo2MSkpCm1kZjwtcmVzaGFwZTI6Om1lbHQobWRmLGlkLnZhcnM9YygiaWQiKSkKbWRmPC1jYmluZChtZGYsc3ViY2xhc3M9KGJvdG5ldF9wcm9iX3Jlc3VsdFsxOjE4MzAsXSRzdWJjbGFzcykpCgpkaWZmPC1nZ3Bsb3QobWRmKSsKICBnZW9tX3RpbGUoYWVzKHg9aWQseT12YXJpYWJsZSxmaWxsPXZhbHVlLHRleHQ9c3ViY2xhc3MpLAogICAgICAgICAgICBjb2xvdXIgPSAid2hpdGUiKSArCiAgc2NhbGVfZmlsbF9ncmFkaWVudG4oY29sb3Vycz1jKCJyZWQiLCJ3aGl0ZSIsImdyZWVuIiksCiAgICAgICAgICAgdmFsdWVzICA9IHJlc2NhbGUoYyhtaW4obWRmJHZhbHVlKSwgMC4wNSwgbWF4KG1kZiR2YWx1ZSkpKSkrCiAgICAgICAgICAgZ3VpZGVzKGZpbGw9RkFMU0UpK3RoZW1lX2J3KCkgIApnZ3Bsb3RseShkaWZmKQpkM2hlYXRtYXAocmZfbSAtIGtubl9tLGNvbG9ycyA9ICJCbHVlcyIsY2VsbG5vdGU9bWF0cml4KGJvdG5ldF9wcm9iX3Jlc3VsdFsxOjE4MzAsXSRzdWJjbGFzcyxuY29sPTMwLG5yb3c9NjEpKQpgYGAKCiMjIyBTdWJjbGFzcyBwcm9iYWJpbGl0eSBhbmFsaXN5cyAoQXR0ZW1wdCBJKQpLTk4gdnMgUkYuCmBgYHtyfQphPW1kZiAlPiUgZmlsdGVyKHZhbHVlPCAtMC4wOSkgJT4lIGdyb3VwX2J5KHN1YmNsYXNzKSAlPiUgc3VtbWFyaXNlKHRvdGxlc3MwMDk9bigpKSAlPiUgYXJyYW5nZShkZXNjKHRvdGxlc3MwMDkpKQpiPW1kZiAgJT4lIGdyb3VwX2J5KHN1YmNsYXNzKSAlPiUgc3VtbWFyaXNlKHRvdGFsPW4oKSkgJT4lIGFycmFuZ2UoZGVzYyh0b3RhbCkpCgpzdWJjbGFzc19wZXJjZW50X2RpZmY8LWlubmVyX2pvaW4oYSxiLGJ5PSJzdWJjbGFzcyIpICU+JSBtdXRhdGUocGVyY2VudD10b3RsZXNzMDA5L3RvdGFsKSAlPiUgYXJyYW5nZShkZXNjKHRvdGFsKSkgCnN1YmNsYXNzX3BlcmNlbnRfZGlmZgoKc3ViY2xhc3NfZGV0ZWN0aW9uczwtYm90bmV0X3Byb2JfcmVzdWx0ICU+JSBtdXRhdGUoZGV0ZWN0ZWRfcmY9aWZlbHNlKFJhbWRvbUZvcmVzdD4wLjUsIkJvdG5ldCIsIk5vcm1hbCIpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkZXRlY3RlZF9rbm49aWZlbHNlKEtOTj4wLjUsIkJvdG5ldCIsIk5vcm1hbCIpKSAlPiUgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG11dGF0ZShjb3JyZWN0X3JmPWlmZWxzZShkZXRlY3RlZF9yZj09VHJ1ZUNsYXNzLDEsMCkpJT4lCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG11dGF0ZShjb3JyZWN0X2tubj1pZmVsc2UoZGV0ZWN0ZWRfa25uPT1UcnVlQ2xhc3MsMSwwKSkgJT4lIAogIGdyb3VwX2J5KHN1YmNsYXNzKSAlPiUgc3VtbWFyaXNlKHRvdGFsX2NvcnJlY3RfcmY9c3VtKGNvcnJlY3RfcmYpLHRvdGFsX2NvcnJlY3Rfa25uPXN1bShjb3JyZWN0X2tubikpCgoKaW5uZXJfam9pbihzdWJjbGFzc19kZXRlY3Rpb25zLHN1YmNsYXNzX3BlcmNlbnRfZGlmZixieT0ic3ViY2xhc3MiKQoKbWRmPC1jYmluZChtZGYscmY9Ym90bmV0X3Byb2JfcmVzdWx0JFJhbWRvbUZvcmVzdFsxOjE4MzBdLGtubj1ib3RuZXRfcHJvYl9yZXN1bHQkS05OWzE6MTgzMF0sdHJ1ZWNsYXNzPWJvdG5ldF9wcm9iX3Jlc3VsdCRUcnVlQ2xhc3NbMToxODMwXSkKYm90bmV0X3Byb2JfcmVzdWx0Cm1kZiAlPiUgZmlsdGVyKHZhbHVlPCAtMC4wOSkgJT4lIGZpbHRlcih0cnVlY2xhc3M9PSJCb3RuZXQiKQptZGYgJT4lIGZpbHRlcih2YWx1ZT4gMC4wOSkgJT4lIGZpbHRlcih0cnVlY2xhc3M9PSJOb3JtYWwiKQoKCmBgYAojIyMgQ2x1c3RlcmluZyBhbmQgUENBCmBgYHtyIGNsdXN0ZXJpbmd9CgprbWVhbnNfbW9kPC1rbWVhbnModGVzdGluZ1ssMToxMF0sY2VudGVycyA9IDEwKQp0ZXN0aW5nX2NsdXN0ZXI8LWNiaW5kKHRlc3RpbmcsY2x1c3Rlcj1rbWVhbnNfbW9kJGNsdXN0ZXIpCnBjYTwtcHJjb21wKHRlc3RpbmdbLGMoLTExLC0xMildLCBjZW50ZXIgPSBUUlVFLCBzY2FsZS4gPSBUUlVFKSAKcGNhX3Rlc3Rpbmc8LWRhdGEuZnJhbWUocGNhJHgsY2xhc3M9dGVzdGluZ19jbHVzdGVyJGNsYXNzLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdWJjbGFzcz10ZXN0aW5nX2NsdXN0ZXIkc3ViY2xhc3MsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjbHVzdGVyPXRlc3RpbmdfY2x1c3RlciRjbHVzdGVyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICApCmc8LWdncGxvdChwY2FfdGVzdGluZyxhZXMoeD1QQzEseT1QQzIpKSsKICBnZW9tX2ppdHRlcihhZXMoY29sb3I9YXMuZmFjdG9yKHN1YmNsYXNzKSx0ZXh0PWNsdXN0ZXIsc2hhcGU9Y2xhc3MpKSsKICAjZ2VvbV9wb2ludChhZXMoc2hhcGU9YXNpZ25hY2lvbiksc2l6ZT0zKSsKICB5bGFiKCJQQzEiKSt4bGFiKCJQQzIiKSsKICB0aGVtZV9jbGFzc2ljKCkrCiNzY2FsZV9zaGFwZV9tYW51YWwodmFsdWVzPWMoOCw2KSkrCiAgIGd1aWRlcyhjb2xvcj1GQUxTRSxhbHBoYT1GQUxTRSkKZ2dwbG90bHkoZykKCmBgYAoKYGBge3J9CiNOb3JtYWwgcHJvYmFiaWxpc3RpYyB0YWJsZQpub3JtYWxfcHJvYl9yZXN1bHQgPSBkYXRhLmZyYW1lKHRlc3RpbmckY2xhc3MsIHByZWRzcmZwcm9icyROb3JtYWwsIGtublByZWRpY3QkTm9ybWFsLCBsb2dpY1JQcmVkaWN0JE5vcm1hbCwgbmFpdmVCYXllc1ByZWRpY3QkTm9ybWFsICxzdm1QcmVkaWN0JE5vcm1hbCkKbmFtZXMobm9ybWFsX3Byb2JfcmVzdWx0KSA9IGMoJ1RydWUgQ2xhc3MnLCdSYW1kb20gRm9yZXN0JywnS05OJywnTG9naXN0aWMgUmVncmVzc2lvbicsICdOYWl2ZSBCYXllcycsICdTdXBvcnQgVk0nKQpub3JtYWxfcHJvYl9yZXN1bHQKYGBgCgo=